{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44fbfab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#抓取数据 完整版3\n",
    "import requests\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "\n",
    "# 设置请求头\n",
    "headers = {\n",
    "    \"Accept\": \"application/json, text/plain, */*\",\n",
    "    \"Accept-Encoding\": \"gzip, deflate, br, zstd\",\n",
    "    \"Accept-Language\": \"zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6\",\n",
    "    \"Client-Version\": \"v2.47.69\",\n",
    "    \"Cookie\": \"_s_tentry=-; Apache=7189147422434.545.1748520876739; SINAGLOBAL=7189147422434.545.1748520876739; ULV=1748520876740:1:1:1:7189147422434.545.1748520876739:; SCF=Am3ezcllLXGaIlKWM5BixhZayhZGksyNNA1ToG1Mx9HKcrYgUod8CWLMHJyAA2Hs05a7CGvWOtAwL6JV_YCKSsY.; SUB=_2A25FPD-4DeRhGeNJ6FQY9S7LyjqIHXVmMD1wrDV8PUNbmtANLRHjkW9NS8FwfEzCt0yRPFIOmo2nLSgcOpIqPE-e; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WFYEqzzDD6OgY1QUPwxssGV5NHD95QfS0ec1K-7S02cWs4DqcjMi--NiK.Xi-2Ri--ciKnRi-zNSKM0So.fehMpSntt; ALF=02_1751112936; XSRF-TOKEN=z9v0J58LTqwSJHuxbhmUrvgs; WBPSESS=ihNrCUzIK5bMw3CG01-xjRiYg_zMKozVwaIClGCK1pOxv0g-W69ZWyZCmNHC7zQsq9refFNnw_JQBrBb1tcQKtbMz0-bzLG9SCp6OG5pEaHIaifO30vjINQxnEcJ0jZl6F7jKISpNt4LzsZijE6Cfw==\",\n",
    "    \"Priority\": \"u=1, i\",\n",
    "    \"Referer\": \"https://weibo.com/2656274875/PtZb7AkOV\",\n",
    "    \"Sec-CH-UA\": \"\\\"Chromium\\\";v=\\\"136\\\", \\\"Microsoft Edge\\\";v=\\\"136\\\", \\\"Not.A/Brand\\\";v=\\\"99\\\"\",\n",
    "    \"Sec-CH-UA-Mobile\": \"?0\",\n",
    "    \"Sec-CH-UA-Platform\": \"\\\"Windows\\\"\",\n",
    "    \"Sec-Fetch-Dest\": \"empty\",\n",
    "    \"Sec-Fetch-Mode\": \"cors\",\n",
    "    \"Sec-Fetch-Site\": \"same-origin\",\n",
    "    \"Server-Version\": \"v2025.05.29.1\",\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0\",\n",
    "    \"X-Requested-With\": \"XMLHttpRequest\",\n",
    "    \"X-XSRF-TOKEN\": \"z9v0J58LTqwSJHuxbhmUrvgs\"\n",
    "}\n",
    "\n",
    "# 定义目标 URL，替换为实际的微博评论 API 链接\n",
    "url = \"https://weibo.com/ajax/statuses/buildComments?is_reload=1&id=5171466858659845&is_show_bulletin=2&is_mix=0&count=10&uid=2656274875&fetch_level=0&locale=zh-CN\"\n",
    "\n",
    "# 设置 max_id 为 None（第一页）\n",
    "max_id = None\n",
    "all_comments = []\n",
    "\n",
    "# 获取并分页获取评论\n",
    "while True:\n",
    "    # 在 URL 中添加 max_id 参数\n",
    "    if max_id:\n",
    "        page_url = f\"{url}&max_id={max_id}\"\n",
    "    else:\n",
    "        page_url = url\n",
    "    \n",
    "    # 发送 GET 请求\n",
    "    response = requests.get(page_url, headers=headers)\n",
    "\n",
    "    # 检查响应状态\n",
    "    if response.status_code == 200:\n",
    "        comments_data = response.json()\n",
    "\n",
    "        # 获取当前页面的评论\n",
    "        comments = comments_data.get(\"data\", [])\n",
    "        all_comments.extend(comments)\n",
    "\n",
    "        # 获取下一页的 max_id（如果有的话）\n",
    "        max_id = comments_data.get(\"max_id\", None)\n",
    "\n",
    "        # 如果 max_id 为空，表示没有更多评论，退出循环\n",
    "        if not max_id:\n",
    "            break\n",
    "    else:\n",
    "        print(f\"请求失败，状态码: {response.status_code}\")\n",
    "        break\n",
    "\n",
    "# 格式化评论时间并保存数据\n",
    "comments_list = []\n",
    "\n",
    "# 将评论数据整理为表格格式\n",
    "for comment in all_comments:\n",
    "    # 获取评论的时间并转换为更易读的格式\n",
    "    created_at = comment.get(\"created_at\", \"未知时间\")\n",
    "    #try:\n",
    "    #   # 将时间字符串转换为标准时间格式\n",
    "    #    created_at = datetime.strptime(created_at, \"%a %b %d %H:%M:%S +0800 %Y\")\n",
    "    #    created_at = created_at.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    #except ValueError:\n",
    "    #    created_at = \"时间格式错误\"\n",
    "\n",
    "    text = comment.get(\"text\", \"没有评论内容\")\n",
    "    user_name = comment['user'].get('screen_name', '未知用户')\n",
    "    like_counts = comment.get(\"like_counts\", 0)\n",
    "\n",
    "    comments_list.append([created_at, user_name, text, like_counts])\n",
    "\n",
    "# 设置保存路径（例如 D:/weibo_comments.csv）\n",
    "save_path = \"D:/weibo_comments.csv\"  # 修改为你希望保存的路径\n",
    "\n",
    "# 使用 pandas 将评论数据保存为 CSV 文件\n",
    "df = pd.DataFrame(comments_list, columns=[\"评论时间\", \"评论者\", \"评论内容\", \"点赞数\"])\n",
    "\n",
    "# 保存为 CSV 文件\n",
    "df.to_csv(save_path, index=False, encoding=\"utf-8-sig\")\n",
    "\n",
    "print(f\"评论数据已保存到 {save_path}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7744c9dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#数据预处理（评论清理、时间转换、表情填充文字）【新版】\n",
    "import pandas as pd\n",
    "import re\n",
    "from datetime import datetime\n",
    "\n",
    "# 表情符号与文字的映射关系\n",
    "emoji_dict = {\n",
    "    \"[威武]\": \"威武\",\n",
    "    \"[666]\": \"666\",\n",
    "    \"[赞]\": \"赞\",\n",
    "    \"[打call]\": \"打call\",\n",
    "    \"[送花花]\": \"送花花\",\n",
    "    \"[good]\": \"good\",\n",
    "    \"[素龙比心]\": \"素龙比心\",\n",
    "    \"[相爱]\": \"相爱\",\n",
    "    \"[努力]\": \"努力\",\n",
    "    \"[awsl]\": \"awsl\",\n",
    "    \"[鼓掌]\": \"鼓掌\",\n",
    "    \"[酷]\": \"酷\",\n",
    "}\n",
    "\n",
    "# 表情替换：提取<img>标签中的alt并替换\n",
    "def replace_emojis_with_alt(text):\n",
    "    # 替换<img ... alt=\"[表情]\"...>为[表情]\n",
    "    return re.sub(r'<img[^>]*alt=\"(\\[.*?\\])\"[^>]*>', r'\\1', text)\n",
    "\n",
    "# 清理评论内容\n",
    "def clean_text(text):\n",
    "    if not isinstance(text, str):\n",
    "        return \"无效内容\"\n",
    "    \n",
    "    # 保留超链接中文文本（如微博话题、用户）\n",
    "    text = re.sub(r'<a[^>]*>(.*?)</a>', r'\\1', text)\n",
    "\n",
    "    # 提取alt里的表情标记\n",
    "    text = replace_emojis_with_alt(text)\n",
    "\n",
    "    # 替换表情标记为中文\n",
    "    for emoji, word in emoji_dict.items():\n",
    "        text = text.replace(emoji, word)\n",
    "\n",
    "    # 删除剩余HTML标签\n",
    "    text = re.sub(r'<.*?>', '', text)\n",
    "\n",
    "    return text.strip()\n",
    "\n",
    "# 评论时间格式转换\n",
    "def convert_time(time_str):\n",
    "    try:\n",
    "        time_obj = datetime.strptime(time_str, \"%a %b %d %H:%M:%S +0800 %Y\")\n",
    "        return time_obj.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    except Exception:\n",
    "        return \"时间格式错误\"\n",
    "\n",
    "# 读取原始CSV\n",
    "df = pd.read_csv(\"D:/weibo_comments.csv\")\n",
    "\n",
    "# 清洗内容列\n",
    "df['评论内容'] = df['评论内容'].apply(clean_text)\n",
    "\n",
    "# 清洗时间列\n",
    "df['评论时间'] = df['评论时间'].apply(convert_time)\n",
    "\n",
    "# 处理空内容\n",
    "df['评论内容'] = df['评论内容'].replace(\"\", \"没有文字内容\")\n",
    "\n",
    "# 查看前几行结果\n",
    "print(df.head())\n",
    "\n",
    "# 保存处理后的结果\n",
    "df.to_csv(\"D:/weibo_comments_cleaned.csv\", index=False, encoding=\"utf-8-sig\")\n",
    "print(\"✅ 数据预处理完成，已保存到 D:/weibo_comments_cleaned.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
